Day 1: Sequence analysis using TraMineR - EDA
Day 2: Sequence analysis using TraMineR - Matching, clustering
Day 3: Association rule mining using priori algorithm
Loading libaries
library(readr)
library(ggplot2)
library(data.table)
library(TraMineR)
library(dplyr)
library(tidyr)
library(TraMineR)
data(mvad)
seqstatl(mvad[, 17:86])
[1] "employment" "FE" "HE" "joblessness" "school" "training"
mvad.alphabet <- c("employment", "FE", "HE", "joblessness", "school",
"training")
mvad.labels <- c("employment", "further education", "higher education",
"joblessness", "school", "training")
mvad.scodes <- c("EM", "FE", "HE", "JL", "SC", "TR")
mvad.seq <- seqdef(mvad, 17:86, alphabet = mvad.alphabet, states = mvad.scodes,
labels = mvad.labels, xtstep = 6)
[>] state coding:
[alphabet] [label] [long label]
1 employment EM employment
2 FE FE further education
3 HE HE higher education
4 joblessness JL joblessness
5 school SC school
6 training TR training
[>] 712 sequences in the data set
[>] min/max sequence length: 70/70
seqiplot(mvad.seq, with.legend = FALSE, title= "Index plot (10 first sequences")
[!!] In seqiplot() : title is deprecated, use main instead.


seqfplot(mvad.seq, withlegend = F, title = "Sequence frequency plot bar width proportional to the frequencies")
[!!] In seqfplot() : title is deprecated, use main instead.
[!!] In seqfplot() : withlegend is deprecated, use with.legend instead.

seqdplot(mvad.seq, withlegend = F, title = "State distribution plot")
[!!] In seqdplot() : title is deprecated, use main instead.
[!!] In seqdplot() : withlegend is deprecated, use with.legend instead.

Import log data
lasi21_logdata <- read_csv("lasi21_logdata.csv")
── Column specification ─────────────────────────────────────────────────────────────────────
cols(
id = col_double(),
sas_id_site = col_double(),
site_type = col_character(),
date_time = col_datetime(format = ""),
spent_time = col_double(),
action = col_character(),
instancename = col_character(),
`avg score` = col_double(),
PassFlag = col_double()
)
lasi21_logdata$site_type <- ifelse(grepl("ssignment", lasi21_logdata$instancename), "assignment",lasi21_logdata$site_type )
lasi21_logdata$site_type <- ifelse(grepl("exam", lasi21_logdata$instancename), "assignment",lasi21_logdata$site_type )
lasi21_logdata <- as.data.table(lasi21_logdata[,c("id","date_time","spent_time","site_type","instancename","PassFlag","avg score")])
head(lasi21_logdata)
Data pre-processing
# convert ms to minutes
lasi21_logdata$spent_time_m <- round(lasi21_logdata$spent_time/60000, digits=1)
# filter out all spent_time < 6s
lasi21_logdata2 <- lasi21_logdata %>% filter(spent_time>6000)
# create a flag for session break (the first click per student)
lasi21_logdata2$session_flag = 0
lasi21_logdata2 <- lasi21_logdata2 %>% arrange(id,date_time,spent_time) %>%
group_by(id) %>%
mutate(session_flag = +(row_number() %in% 1))
# create a flag for session break (aka where time spent > 30 minutes)
lasi21_logdata2$session_flag <- ifelse(lasi21_logdata2$spent_time_m>30, 1, lasi21_logdata2$session_flag)
# create session number
lasi21_logdata2$session_num = cumsum(lasi21_logdata2$session_flag)
# remove session break
lasi21_logdata2 <- as.data.table(lasi21_logdata2)
lasi21_logdata2 <- lasi21_logdata2[session_flag!=1,]
# for each learning session, calculate the cummulative time spent
lasi21_logdata2 <- lasi21_logdata2 %>%
arrange(id,date_time,spent_time) %>%
group_by(session_num) %>%
mutate(spent_time_m_cum = cumsum(spent_time_m))
# create time unit as 1/10 of a minute (6s)
lasi21_logdata2$time_unit <- round(lasi21_logdata2$spent_time_m_cum*10,digits=0)
lasi21_logdata_session <- lasi21_logdata2 %>% group_by(session_num) %>% top_n(1, spent_time_m_cum)
hist(lasi21_logdata_session$spent_time_m_cum, main="Learning session length", xlab = "Minutes", breaks=200)

# Remove learning session < 5 mins
lasi21_logdata2 <- merge(lasi21_logdata2,lasi21_logdata_session[lasi21_logdata_session$spent_time_m_cum>5 & lasi21_logdata_session$spent_time_m_cum<120, c("session_num")])
Define sequences
# define sequences columns
log_sts.seq <- seqdef(log_sts,2:1200)
[>] found missing values ('NA') in sequence data
[>] preparing 18602 sequences
[>] coding void elements with '%' and missing values with '*'
[>] 12 distinct states appear in the data:
1 = assignment
2 = collaborate
3 = content
4 = forumng
5 = glossary
6 = homepage
7 = questionnaire
8 = resource
9 = studio
10 = subpage
11 = url
12 = wiki
[>] state coding:
[alphabet] [label] [long label]
1 assignment assignment assignment
2 collaborate collaborate collaborate
3 content content content
4 forumng forumng forumng
5 glossary glossary glossary
6 homepage homepage homepage
7 questionnaire questionnaire questionnaire
8 resource resource resource
9 studio studio studio
10 subpage subpage subpage
11 url url url
12 wiki wiki wiki
[>] 18602 sequences in the data set
[>] min/max sequence length: 51/1199
EDA
# plot the first 10 sequences with length=100
layout(matrix(c(1,1,1,2), nrow=1, byrow=TRUE))
seqiplot(log_sts.seq[611:631,1:100], with.legend = F, main = "Index plot (10 first sequences)")
seqlegend(log_sts.seq)


# State distribution plot
layout(matrix(c(1,1,1,2), nrow=1, byrow=TRUE))
seqdplot(log_sts.seq[,1:200], main = "State distribution plot", with.legend = FALSE)
seqlegend(log_sts.seq)

# Sequence frequency plot
layout(matrix(c(1,1,1,2), nrow=1, byrow=TRUE))
seqfplot(log_sts.seq[,1:200], main = "Sequence frequency plot", with.legend = FALSE, pbarw = TRUE)
seqlegend(log_sts.seq)

# mean time by state
layout(matrix(c(1,1,1,2), nrow=1, byrow=TRUE))
seqmtplot(log_sts.seq[,1:200], main = "Mean time plot", with.legend = FALSE)
seqlegend(log_sts.seq)

# Stability within sequences
# Shannon's entropy as a measure of the diversity of states observed at the considered time point
# It equals 0 when all cases are in the same state (it is thus easy to predict in which state an individual is)
# It is maximum when the cases are equally distributed between the states
seqHtplot(log_sts.seq[,1:200], title = "Entropy index")
[!!] In seqHtplot() : title is deprecated, use main instead.

# Turbulence
Turbulence <- seqST(log_sts.seq[,1:200])
summary(Turbulence)
Turbulence
Min. : 1.000
1st Qu.: 2.000
Median : 4.038
Mean : 4.704
3rd Qu.: 6.641
Max. :32.312
hist(Turbulence, col = "cyan", main = "Sequence turbulence",breaks=50)

# Transitions of events
# define seq transitions
log_sts.seqe <- seqecreate(log_sts.seq[,1:200])
# find frequent subsequences
fsubseq <- seqefsub(log_sts.seqe, pMinSupport = 0.05)
[!!] In seqefsub() : pMinSupport is deprecated, use pmin.support instead.
# plot 15 most frequent subsquences
plot(fsubseq[1:15], col = "cyan", main="Top 15 frequent subsequences")

Sequence similarities and clustering
# Compute sequence distances using OM with transition rate as substitution cost
log_sts.seq.om1 <- round(seqdist(log_sts.seq[1:1000,1:200], method = "OM", indel = 1, sm = "CONSTANT"),2)
[>] 1000 sequences with 12 distinct states
[>] Computing sm with seqcost using CONSTANT
[>] creating 12x12 substitution-cost matrix using 2 as constant value
[>] 846 distinct sequences
[>] min/max sequence lengths: 51/200
[>] computing distances using the OM metric
[>] elapsed time: 28.993 secs


fviz_nbclust(log_sts.seq.om1, FUN = hcut, method = "silhouette")+
labs(subtitle = "Silhouette method")

cluster3 <- cutree(clusterward, k = 3)
cluster3 <- factor(HC3c, labels = paste("Type", 1:3))
table(cluster3)
cluster3
Type 1 Type 2 Type 3
442 291 267
# frequency plot of sequences by cluster
seqfplot(log_sts.seq[1:1000,1:200], group = cluster3, pbarw = T)

seqmtplot(log_sts.seq[1:1000,1:200], group = cluster3)

# layout(matrix(c(1,1,1,2), nrow=1, byrow=TRUE))
seqdplot(log_sts.seq[1:1000,1:200], group = cluster3, border = NA, with.legend = T)

# seqlegend(log_sts.seq)
covar$grade <- rnorm(1000, mean=70, sd=10); covar$grade <- covar$grade[covar$grade > 1 & covar$grade < 100]
Error in `$<-.data.frame`(`*tmp*`, grade, value = c(72.1121413037559, :
replacement has 998 rows, data has 1000
reglog <- list()
for (i in 1:length(levels(cluster3))) {
reglog[[i]] <- glm((cluster3 == levels(cluster3)[i]) ~ condition + grade + surveymetric1 + surveymetric2, family = "binomial", data = covar)}
tbls <- list()
for (i in 1:length(levels(cluster3))) {tbls[[i]] <- tbl_regression(reglog[[i]], exponentiate = TRUE)}
tbl_merge(
tbls = tbls,
tab_spanner = c("**Type 1**", "**Type 2**", "**Type 3**")
)
| Characteristic |
Type 1
|
Type 2
|
Type 3
|
| OR |
95% CI |
p-value |
OR |
95% CI |
p-value |
OR |
95% CI |
p-value |
| condition |
|
|
|
|
|
|
|
|
|
| Control |
— |
— |
|
— |
— |
|
— |
— |
|
| Treatment |
0.90 |
0.70, 1.16 |
0.4 |
1.15 |
0.87, 1.51 |
0.3 |
0.99 |
0.75, 1.31 |
>0.9 |
| grade |
1.00 |
0.99, 1.02 |
0.5 |
0.99 |
0.98, 1.01 |
0.4 |
1.00 |
0.99, 1.02 |
0.8 |
| surveymetric1 |
0.99 |
0.71, 1.36 |
>0.9 |
0.87 |
0.61, 1.24 |
0.4 |
1.18 |
0.82, 1.69 |
0.4 |
| surveymetric2 |
0.98 |
0.80, 1.19 |
0.8 |
0.99 |
0.80, 1.23 |
>0.9 |
1.04 |
0.83, 1.30 |
0.7 |
Comparison of sequences

LS0tCnRpdGxlOiAiVGVtcG9yYWwgYW5kIHNlcXVlbnRpYWwgYW5hbHlzaXMgZm9yIGxlYXJuaW5nIGFuYWx5dGljcyIKYXV0aG9yOiAiUXVhbiBOZ3V5ZW4sIFBvc3Rkb2N0b3JhbCBGZWxsb3csIFNjaG9vbCBvZiBJbmZvcm1hdGlvbiwgVW5pdmVyc2l0eSBvZiBNaWNoaWdhbiIKb3V0cHV0OiBodG1sX25vdGVib29rCi0tLQoKCiMgRGF5IDE6IFNlcXVlbmNlIGFuYWx5c2lzIHVzaW5nIFRyYU1pbmVSIC0gRURBCiMgRGF5IDI6IFNlcXVlbmNlIGFuYWx5c2lzIHVzaW5nIFRyYU1pbmVSIC0gTWF0Y2hpbmcsIGNsdXN0ZXJpbmcKIyBEYXkgMzogQXNzb2NpYXRpb24gcnVsZSBtaW5pbmcgdXNpbmcgcHJpb3JpIGFsZ29yaXRobQoKIyMgTG9hZGluZyBsaWJhcmllcwoKYGBge3J9CmxpYnJhcnkocmVhZHIpCmxpYnJhcnkoZ2dwbG90MikKbGlicmFyeShkYXRhLnRhYmxlKQpsaWJyYXJ5KFRyYU1pbmVSKQpsaWJyYXJ5KGRwbHlyKQpsaWJyYXJ5KHRpZHlyKQpgYGAKCmBgYHtyfQpsaWJyYXJ5KFRyYU1pbmVSKQpkYXRhKG12YWQpCnNlcXN0YXRsKG12YWRbLCAxNzo4Nl0pCm12YWQuYWxwaGFiZXQgPC0gYygiZW1wbG95bWVudCIsICJGRSIsICJIRSIsICJqb2JsZXNzbmVzcyIsICJzY2hvb2wiLCAKICAgICJ0cmFpbmluZyIpCm12YWQubGFiZWxzIDwtIGMoImVtcGxveW1lbnQiLCAiZnVydGhlciBlZHVjYXRpb24iLCAiaGlnaGVyIGVkdWNhdGlvbiIsIAogICAgImpvYmxlc3NuZXNzIiwgInNjaG9vbCIsICJ0cmFpbmluZyIpCm12YWQuc2NvZGVzIDwtIGMoIkVNIiwgIkZFIiwgIkhFIiwgIkpMIiwgIlNDIiwgIlRSIikKbXZhZC5zZXEgPC0gc2VxZGVmKG12YWQsIDE3Ojg2LCBhbHBoYWJldCA9IG12YWQuYWxwaGFiZXQsIHN0YXRlcyA9IG12YWQuc2NvZGVzLCAKICAgIGxhYmVscyA9IG12YWQubGFiZWxzLCB4dHN0ZXAgPSA2KQoKYGBgCmBgYHtyfQpoZWFkKG12YWQpCmBgYAoKYGBge3J9CnNlcWlwbG90KG12YWQuc2VxLCB3aXRoLmxlZ2VuZCA9IEZBTFNFLCB0aXRsZT0gIkluZGV4IHBsb3QgKDEwIGZpcnN0IHNlcXVlbmNlcyIpCmBgYApgYGB7cn0Kc2VxSXBsb3QobXZhZC5zZXEsIHNvcnR2ID0gImZyb20uc3RhcnQiLCB3aXRoLmxlZ2VuZCA9IEZBTFNFKQpgYGAKYGBge3J9CnNlcWZwbG90KG12YWQuc2VxLCB3aXRobGVnZW5kID0gRiwgdGl0bGUgPSAiU2VxdWVuY2UgZnJlcXVlbmN5IHBsb3QgYmFyIHdpZHRoIHByb3BvcnRpb25hbCB0byB0aGUgZnJlcXVlbmNpZXMiKQpgYGAKYGBge3J9CiBzZXFkcGxvdChtdmFkLnNlcSwgd2l0aGxlZ2VuZCA9IEYsICB0aXRsZSA9ICJTdGF0ZSBkaXN0cmlidXRpb24gcGxvdCIpCmBgYAoKCiMgSW1wb3J0IGxvZyBkYXRhCgpgYGB7cn0KbGFzaTIxX2xvZ2RhdGEgPC0gcmVhZF9jc3YoImxhc2kyMV9sb2dkYXRhLmNzdiIpCmxhc2kyMV9sb2dkYXRhJHNpdGVfdHlwZSA8LSBpZmVsc2UoZ3JlcGwoInNzaWdubWVudCIsIGxhc2kyMV9sb2dkYXRhJGluc3RhbmNlbmFtZSksICJhc3NpZ25tZW50IixsYXNpMjFfbG9nZGF0YSRzaXRlX3R5cGUgKQpsYXNpMjFfbG9nZGF0YSRzaXRlX3R5cGUgPC0gaWZlbHNlKGdyZXBsKCJleGFtIiwgbGFzaTIxX2xvZ2RhdGEkaW5zdGFuY2VuYW1lKSwgImFzc2lnbm1lbnQiLGxhc2kyMV9sb2dkYXRhJHNpdGVfdHlwZSApCgpsYXNpMjFfbG9nZGF0YSA8LSBhcy5kYXRhLnRhYmxlKGxhc2kyMV9sb2dkYXRhWyxjKCJpZCIsImRhdGVfdGltZSIsInNwZW50X3RpbWUiLCJzaXRlX3R5cGUiLCJpbnN0YW5jZW5hbWUiLCJQYXNzRmxhZyIsImF2ZyBzY29yZSIpXSkKaGVhZChsYXNpMjFfbG9nZGF0YSkKYGBgCgojIERhdGEgcHJlLXByb2Nlc3NpbmcKYGBge3J9CiMgY29udmVydCBtcyB0byBtaW51dGVzCmxhc2kyMV9sb2dkYXRhJHNwZW50X3RpbWVfbSA8LSByb3VuZChsYXNpMjFfbG9nZGF0YSRzcGVudF90aW1lLzYwMDAwLCBkaWdpdHM9MSkKCiMgZmlsdGVyIG91dCBhbGwgc3BlbnRfdGltZSA8IDZzCmxhc2kyMV9sb2dkYXRhMiA8LSBsYXNpMjFfbG9nZGF0YSAlPiUgZmlsdGVyKHNwZW50X3RpbWU+NjAwMCkKCiMgY3JlYXRlIGEgZmxhZyBmb3Igc2Vzc2lvbiBicmVhayAodGhlIGZpcnN0IGNsaWNrIHBlciBzdHVkZW50KQpsYXNpMjFfbG9nZGF0YTIkc2Vzc2lvbl9mbGFnID0gMApsYXNpMjFfbG9nZGF0YTIgPC0gbGFzaTIxX2xvZ2RhdGEyICU+JSBhcnJhbmdlKGlkLGRhdGVfdGltZSxzcGVudF90aW1lKSAlPiUgCiAgICBncm91cF9ieShpZCkgJT4lCiAgICBtdXRhdGUoc2Vzc2lvbl9mbGFnICA9ICsocm93X251bWJlcigpICVpbiUgMSkpCgojIGNyZWF0ZSBhIGZsYWcgZm9yIHNlc3Npb24gYnJlYWsgKGFrYSB3aGVyZSB0aW1lIHNwZW50ID4gMzAgbWludXRlcykKbGFzaTIxX2xvZ2RhdGEyJHNlc3Npb25fZmxhZyA8LSBpZmVsc2UobGFzaTIxX2xvZ2RhdGEyJHNwZW50X3RpbWVfbT4zMCwgMSwgbGFzaTIxX2xvZ2RhdGEyJHNlc3Npb25fZmxhZykKCiMgY3JlYXRlIHNlc3Npb24gbnVtYmVyIApsYXNpMjFfbG9nZGF0YTIkc2Vzc2lvbl9udW0gPSBjdW1zdW0obGFzaTIxX2xvZ2RhdGEyJHNlc3Npb25fZmxhZykKCiMgcmVtb3ZlIHNlc3Npb24gYnJlYWsKbGFzaTIxX2xvZ2RhdGEyIDwtIGFzLmRhdGEudGFibGUobGFzaTIxX2xvZ2RhdGEyKQpsYXNpMjFfbG9nZGF0YTIgPC0gbGFzaTIxX2xvZ2RhdGEyW3Nlc3Npb25fZmxhZyE9MSxdCgoKIyBmb3IgZWFjaCBsZWFybmluZyBzZXNzaW9uLCBjYWxjdWxhdGUgdGhlIGN1bW11bGF0aXZlIHRpbWUgc3BlbnQKbGFzaTIxX2xvZ2RhdGEyIDwtIGxhc2kyMV9sb2dkYXRhMiAlPiUgCiAgICAgICAgICAgICAgICAgICAgICAgIGFycmFuZ2UoaWQsZGF0ZV90aW1lLHNwZW50X3RpbWUpICU+JSAKICAgICAgICAgICAgICAgICAgICAgICAgZ3JvdXBfYnkoc2Vzc2lvbl9udW0pICU+JSAKICAgICAgICAgICAgICAgICAgICAgICAgbXV0YXRlKHNwZW50X3RpbWVfbV9jdW0gPSBjdW1zdW0oc3BlbnRfdGltZV9tKSkKCiMgY3JlYXRlIHRpbWUgdW5pdCBhcyAxLzEwIG9mIGEgbWludXRlICg2cykKbGFzaTIxX2xvZ2RhdGEyJHRpbWVfdW5pdCA8LSByb3VuZChsYXNpMjFfbG9nZGF0YTIkc3BlbnRfdGltZV9tX2N1bSoxMCxkaWdpdHM9MCkKCmxhc2kyMV9sb2dkYXRhX3Nlc3Npb24gPC0gbGFzaTIxX2xvZ2RhdGEyICU+JSBncm91cF9ieShzZXNzaW9uX251bSkgJT4lIHRvcF9uKDEsIHNwZW50X3RpbWVfbV9jdW0pCgpoaXN0KGxhc2kyMV9sb2dkYXRhX3Nlc3Npb24kc3BlbnRfdGltZV9tX2N1bSwgbWFpbj0iTGVhcm5pbmcgc2Vzc2lvbiBsZW5ndGgiLCB4bGFiID0gIk1pbnV0ZXMiLCBicmVha3M9MjAwKQoKIyBSZW1vdmUgbGVhcm5pbmcgc2Vzc2lvbiA8IDUgbWlucwpsYXNpMjFfbG9nZGF0YTIgPC0gbWVyZ2UobGFzaTIxX2xvZ2RhdGEyLGxhc2kyMV9sb2dkYXRhX3Nlc3Npb25bbGFzaTIxX2xvZ2RhdGFfc2Vzc2lvbiRzcGVudF90aW1lX21fY3VtPjUgJiBsYXNpMjFfbG9nZGF0YV9zZXNzaW9uJHNwZW50X3RpbWVfbV9jdW08MTIwLCBjKCJzZXNzaW9uX251bSIpXSkKCmBgYAoKIyBUcmFuc2Zvcm0gZGF0YSBpbnRvIFNUUyBmb3JtYXQKYGBge3J9CiMgY3JlYXRlIGEgZnVuY3Rpb24gdG8gZXhwYW5kIHRpbWUgdW5pdCBmb3IgZWFjaCBsZWFybmluZyBzZXNzaW9uCgpmMSA8LSBmdW5jdGlvbih4MSl7CiAgICB4MSA8LSAxOm1heCh4MSkKICAgIG0xIDwtIG1heChjKGxlbmd0aCh4MSkpKQogICAgbGVuZ3RoKHgxKSA8LSBtMQogICAgbGlzdCh0aW1lX3VuaXQgPSB4MSkKfQoKIyBjcmVhdGUgYSBzdWJzZXQKbGFzaTIxX2xvZ2RhdGEzIDwtIGxhc2kyMV9sb2dkYXRhMlssYygic2Vzc2lvbl9udW0iLCJ0aW1lX3VuaXQiLCJzaXRlX3R5cGUiKV0KCiMgY3JlYXRlIGFuIGV4cGFuZGVkIGRmCnRlc3QgPC0gc2V0RFQobGFzaTIxX2xvZ2RhdGEzKVssIGYxKHRpbWVfdW5pdCksIC4oc2Vzc2lvbl9udW0pXQoKIyBtZXJnZSB3aXRoIGxvZyBkYXRhCnRlc3QgPC0gbWVyZ2UodGVzdCxsYXNpMjFfbG9nZGF0YTMsYnk9Yygic2Vzc2lvbl9udW0iLCJ0aW1lX3VuaXQiKSxhbGwueD1UUlVFKQoKIyBmaWxsIG1pc3NpbmcgdXB3YXJkCnRlc3QgPC0gdGVzdCAlPiUgZmlsbChzaXRlX3R5cGUsLmRpcmVjdGlvbiA9ICJ1cCIpCmBgYAoKCgpgYGB7cn0KIyBTZXF1ZW5jZSBsZW5ndGggZGlzdHJpYnV0aW9uCmhpc3QobGFzaTIxX2xvZ2RhdGEzWyxtYXgodGltZV91bml0KSwgYnk9InNlc3Npb25fbnVtIl0kVjEsIGJyZWFrcz0xMDAsIG1haW49Ikxlbmd0aCBvZiBsZWFybmluZyBzZXNzaW9uIiwgeGxhYj0nU2VxdWVuY2UgbGVuZ3RoJykKYGBgCgpgYGB7cn0KIyBjdW1tdWxhdGl2ZSBwbG90IG9mIHNlcSBsZW5ndGgKcGxvdChlY2RmKGxhc2kyMV9sb2dkYXRhM1ssbWF4KHRpbWVfdW5pdCksIGJ5PSJzZXNzaW9uX251bSJdJFYxKSwgbWFpbiA9J0N1bW11bGF0aXZlIGRpc3RyaWJ1dGlvbiBvZiBzZXEgbGVuZ3RoJykKYGBgCiMgRGVmaW5lIHNlcXVlbmNlcwoKYGBge3J9CiMgY29udmVydCB0byBTVFMgZm9ybWF0ICh3aWRlIGZvcm1hdCkKbG9nX3N0cyA8LSBzcHJlYWQodGVzdCwgdGltZV91bml0LCBzaXRlX3R5cGUpCgojIGRlZmluZSBzZXF1ZW5jZXMgY29sdW1ucwpsb2dfc3RzLnNlcSA8LSBzZXFkZWYobG9nX3N0cywyOjEyMDApCgpgYGAKCiMgRURBCgpgYGB7cn0KIyBwbG90IHRoZSBmaXJzdCAxMCBzZXF1ZW5jZXMgd2l0aCBsZW5ndGg9MTAwCmxheW91dChtYXRyaXgoYygxLDEsMSwyKSwgbnJvdz0xLCBieXJvdz1UUlVFKSkKc2VxaXBsb3QobG9nX3N0cy5zZXFbNjExOjYzMSwxOjEwMF0sIHdpdGgubGVnZW5kID0gRiwgbWFpbiA9ICJJbmRleCBwbG90ICgxMCBmaXJzdCBzZXF1ZW5jZXMpIikKc2VxbGVnZW5kKGxvZ19zdHMuc2VxKQpgYGAKYGBge3J9CiMgUGxvdCAyMDAgc2VxdWVuY2VzIHNvcnRlZCBieSBMQ1MgKGxvbmdlc3QgY29tbW9uIHN1YnNlcXVlbmNlKSBkaXN0YW5jZQpkaXN0Lm1vc3RmcmVxIDwtIHNlcWRpc3QobG9nX3N0cy5zZXEsIG1ldGhvZCA9ICJMQ1MiLCByZWZzZXEgPSAwKQpzZXFJcGxvdChsb2dfc3RzLnNlcVsxOjIwMCwxOjIwMF0sIGJvcmRlciA9IE5BLCBzb3J0diA9IGRpc3QubW9zdGZyZXEsIHdpdGgubGVnZW5kPUYpCmBgYAoKCmBgYHtyfQojIFN0YXRlIGRpc3RyaWJ1dGlvbiBwbG90CmxheW91dChtYXRyaXgoYygxLDEsMSwyKSwgbnJvdz0xLCBieXJvdz1UUlVFKSkKc2VxZHBsb3QobG9nX3N0cy5zZXFbLDE6MjAwXSwgbWFpbiA9ICJTdGF0ZSBkaXN0cmlidXRpb24gcGxvdCIsIHdpdGgubGVnZW5kID0gRkFMU0UpCnNlcWxlZ2VuZChsb2dfc3RzLnNlcSkKYGBgCgpgYGB7cn0KIyBTZXF1ZW5jZSBmcmVxdWVuY3kgcGxvdApsYXlvdXQobWF0cml4KGMoMSwxLDEsMiksIG5yb3c9MSwgYnlyb3c9VFJVRSkpCnNlcWZwbG90KGxvZ19zdHMuc2VxWywxOjIwMF0sIG1haW4gPSAiU2VxdWVuY2UgZnJlcXVlbmN5IHBsb3QiLCB3aXRoLmxlZ2VuZCA9IEZBTFNFLCBwYmFydyA9IFRSVUUpCnNlcWxlZ2VuZChsb2dfc3RzLnNlcSkKYGBgCmBgYHtyfQojIG1lYW4gdGltZSBieSBzdGF0ZQpsYXlvdXQobWF0cml4KGMoMSwxLDEsMiksIG5yb3c9MSwgYnlyb3c9VFJVRSkpCnNlcW10cGxvdChsb2dfc3RzLnNlcVssMToyMDBdLCBtYWluID0gIk1lYW4gdGltZSBwbG90Iiwgd2l0aC5sZWdlbmQgPSBGQUxTRSkKc2VxbGVnZW5kKGxvZ19zdHMuc2VxKQpgYGAKCmBgYHtyfQojIFN0YWJpbGl0eSB3aXRoaW4gc2VxdWVuY2VzCiMgU2hhbm5vbidzIGVudHJvcHkgYXMgYSBtZWFzdXJlIG9mIHRoZSBkaXZlcnNpdHkgb2Ygc3RhdGVzIG9ic2VydmVkIGF0IHRoZSBjb25zaWRlcmVkIHRpbWUgcG9pbnQKIyBJdCBlcXVhbHMgMCB3aGVuIGFsbCBjYXNlcyBhcmUgaW4gdGhlIHNhbWUgc3RhdGUgIChpdCBpcyB0aHVzIGVhc3kgdG8gcHJlZGljdCBpbiB3aGljaCBzdGF0ZSBhbiBpbmRpdmlkdWFsIGlzKQojIEl0IGlzIG1heGltdW0gd2hlbiB0aGUgY2FzZXMgYXJlIGVxdWFsbHkgZGlzdHJpYnV0ZWQgYmV0d2VlbiB0aGUgc3RhdGVzCnNlcUh0cGxvdChsb2dfc3RzLnNlcVssMToyMDBdLCB0aXRsZSA9ICJFbnRyb3B5IGluZGV4IikKYGBgCgpgYGB7cn0KIyBUdXJidWxlbmNlClR1cmJ1bGVuY2UgPC0gc2VxU1QobG9nX3N0cy5zZXFbLDE6MjAwXSkKc3VtbWFyeShUdXJidWxlbmNlKQpoaXN0KFR1cmJ1bGVuY2UsIGNvbCA9ICJjeWFuIiwgbWFpbiA9ICJTZXF1ZW5jZSB0dXJidWxlbmNlIixicmVha3M9NTApCgpgYGAKCmBgYHtyfQojIFRyYW5zaXRpb25zIG9mIGV2ZW50cwoKIyBkZWZpbmUgc2VxIHRyYW5zaXRpb25zCmxvZ19zdHMuc2VxZSA8LSBzZXFlY3JlYXRlKGxvZ19zdHMuc2VxWywxOjIwMF0pCgojIGZpbmQgZnJlcXVlbnQgc3Vic2VxdWVuY2VzCmZzdWJzZXEgPC0gc2VxZWZzdWIobG9nX3N0cy5zZXFlLCBwTWluU3VwcG9ydCA9IDAuMDUpCgojIHBsb3QgMTUgbW9zdCBmcmVxdWVudCBzdWJzcXVlbmNlcwpwbG90KGZzdWJzZXFbMToxNV0sIGNvbCA9ICJjeWFuIiwgbWFpbj0iVG9wIDE1IGZyZXF1ZW50IHN1YnNlcXVlbmNlcyIpCgpgYGAKCiMgU2VxdWVuY2Ugc2ltaWxhcml0aWVzIGFuZCBjbHVzdGVyaW5nCgpgYGB7cn0KIyBDb21wdXRlIHNlcXVlbmNlIGRpc3RhbmNlcyB1c2luZyBPTSB3aXRoIHRyYW5zaXRpb24gcmF0ZSBhcyBzdWJzdGl0dXRpb24gY29zdApsb2dfc3RzLnNlcS5vbTEgPC0gcm91bmQoc2VxZGlzdChsb2dfc3RzLnNlcVsxOjEwMDAsMToyMDBdLCBtZXRob2QgPSAiT00iLCBpbmRlbCA9IDEsIHNtID0gIkNPTlNUQU5UIiksMikKYGBgCmBgYHtyfQojIGFwcGx5IGFnZ2xvbWVyYXRlIGhpZXJhcmNoaWNhbCBjbHVzdGVyaW5nCmxpYnJhcnkoY2x1c3RlcikKY2x1c3RlcndhcmQgPC0gYWduZXMobG9nX3N0cy5zZXEub20xLCBkaXNzID0gVFJVRSwgbWV0aG9kID0gIndhcmQiKQpoY2QgPC0gYXMuZGVuZHJvZ3JhbShjbHVzdGVyd2FyZCkKcGxvdChoY2QsIHdoaWNoLnBsb3RzID0gMiwgbWFpbj0iRGVuZG9ncmFtIG9mIHNlcXVlbmNlcyBjbHVzdGVyaW5nIiwgbGVhZmxhYiA9ICJub25lIikKYGBgCmBgYHtyfQpsaWJyYXJ5KGZhY3RvZXh0cmEpCiMgUGxvdCBzY3JlZSBwbG90IHVzaW5nIHdzcwpmdml6X25iY2x1c3QobG9nX3N0cy5zZXEub20xLCBGVU4gPSBoY3V0LCBtZXRob2QgPSAid3NzIikKYGBgCmBgYHtyfQojIFBsb3QgU2lsaG91ZXR0ZSAKZnZpel9uYmNsdXN0KGxvZ19zdHMuc2VxLm9tMSwgRlVOID0gaGN1dCwgbWV0aG9kID0gInNpbGhvdWV0dGUiKSsKbGFicyhzdWJ0aXRsZSA9ICJTaWxob3VldHRlIG1ldGhvZCIpCmBgYApgYGB7cn0KIyBVc2Ugaz0zIGFzIHRoZSBudW1iZXIgb2YgY2x1c3RlcgpjbHVzdGVyMyA8LSBjdXRyZWUoY2x1c3RlcndhcmQsIGsgPSAzKQpjbHVzdGVyMyA8LSBmYWN0b3IoSEMzYywgbGFiZWxzID0gcGFzdGUoIlR5cGUiLCAxOjMpKQp0YWJsZShjbHVzdGVyMykKYGBgCmBgYHtyfQojIGZyZXF1ZW5jeSBwbG90IG9mIHNlcXVlbmNlcyBieSBjbHVzdGVyCnNlcWZwbG90KGxvZ19zdHMuc2VxWzE6MTAwMCwxOjIwMF0sIGdyb3VwID0gY2x1c3RlcjMsIHBiYXJ3ID0gVCkKYGBgCgpgYGB7cn0KIyBtZWFuIHRpbWUgcGxvdCBvZiBzdGF0ZXMgYnkgY2x1c3RlciBtZW1iZXJzaGlwCnNlcW10cGxvdChsb2dfc3RzLnNlcVsxOjEwMDAsMToyMDBdLCBncm91cCA9IGNsdXN0ZXIzKQpgYGAKYGBge3J9CiMgbGF5b3V0KG1hdHJpeChjKDEsMSwxLDIpLCBucm93PTEsIGJ5cm93PVRSVUUpKQpzZXFkcGxvdChsb2dfc3RzLnNlcVsxOjEwMDAsMToyMDBdLCBncm91cCA9IGNsdXN0ZXIzLCBib3JkZXIgPSBOQSwgd2l0aC5sZWdlbmQgPSBUKQojIHNlcWxlZ2VuZChsb2dfc3RzLnNlcSkKCmBgYAoKYGBge3J9CiMgbWFkZSB1cCBjb3ZhcmlhdGVzIGZvciAxMDAwIHNlcXVlbmNlcyAoZS5nLiwgY29uZGl0aW9uLCBncmFkZSwgc3VydmV5X21ldHJpYzEsIHN1cnZleV9tZXRyaWMyKQpjb3ZhciA8LSBkYXRhLmZyYW1lKG1hdHJpeChuY29sID0gNCwgbnJvdyA9IDEwMDApKQpjb2xuYW1lcyhjb3ZhcikgPC0gYygiY29uZGl0aW9uIiwgImdyYWRlIiwgInN1cnZleW1ldHJpYzEiLCAic3VydmV5bWV0cmljMiIpCgpzZXQuc2VlZCgxMjMpCmNvdmFyJGNvbmRpdGlvbiA8LSBzYW1wbGUoeD1jKCJDb250cm9sIiwiVHJlYXRtZW50IiksMTAwMCxwcm9iPWMoMC41LDAuNSkscmVwbGFjZT1UKQoKc2V0LnNlZWQoMTIzKQpjb3ZhciRncmFkZSA8LSByb3VuZChybm9ybSgxMDAwLCBtZWFuPTYwLCBzZD0xMCksMCkKCnNldC5zZWVkKDEyMykKY292YXIkc3VydmV5bWV0cmljMSA8LSBzYW1wbGUoeD0xOjUsIHByb2IgPSBjKDAuMSwwLjIsMC40LDAuMiwwLjEpLCByZXBsYWNlID0gVCkKCnNldC5zZWVkKDEyMykKY292YXIkc3VydmV5bWV0cmljMiA8LSBzYW1wbGUoeD0xOjUsIHByb2IgPSBjKDAuMSwwLjIsMC4zLDAuMywwLjEpLCByZXBsYWNlID0gVCkKYGBgCgpgYGB7cn0KbGlicmFyeShndHN1bW1hcnkpCgojIHJ1biBsb2dpc3RpYyByZWdyZXNzaW9uIGZvciBlYWNoIGNsdXN0ZXIgb2Ygc2VxdWVuY2VzCnJlZ2xvZyA8LSBsaXN0KCkKZm9yIChpIGluIDE6bGVuZ3RoKGxldmVscyhjbHVzdGVyMykpKSB7cmVnbG9nW1tpXV0gPC0gZ2xtKChjbHVzdGVyMyA9PSBsZXZlbHMoY2x1c3RlcjMpW2ldKSB+IAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb25kaXRpb24gKyBncmFkZSArIHN1cnZleW1ldHJpYzEgKyBzdXJ2ZXltZXRyaWMyLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZmFtaWx5ID0gImJpbm9taWFsIiwgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGEgPSBjb3Zhcil9CiMgY3JlYXRlIG5pY2Ugc3VtbWFyeSBvdXRwdXQgdXNpbmcgZ3RzdW1tYXJ5IHBhY2thZ2UgCiMgaHR0cDovL3d3dy5kYW5pZWxkc2pvYmVyZy5jb20vZ3RzdW1tYXJ5L2FydGljbGVzL3RibF9yZWdyZXNzaW9uLmh0bWwKdGJscyA8LSBsaXN0KCkKZm9yIChpIGluIDE6bGVuZ3RoKGxldmVscyhjbHVzdGVyMykpKSB7dGJsc1tbaV1dIDwtIHRibF9yZWdyZXNzaW9uKHJlZ2xvZ1tbaV1dLCBleHBvbmVudGlhdGUgPSBUUlVFKX0KCnRibF9tZXJnZSgKICAgIHRibHMgPSB0YmxzLAogICAgdGFiX3NwYW5uZXIgPSBjKCIqKlR5cGUgMSoqIiwgIioqVHlwZSAyKioiLCAiKipUeXBlIDMqKiIpCiAgKQpgYGAKCiMgQ29tcGFyaXNvbiBvZiBzZXF1ZW5jZXMKCmBgYHtyfQojIFBsb3Qgc2VxdWVuY2VzIGJ5IGdyb3VwCmRpc3QucmVmc2VxIDwtIHNlcWRpc3QobG9nX3N0cy5zZXFbMToxMDAwLDE6MjAwXSwgcmVmc2VxID0gMCwgbWV0aG9kID0gIkxDUyIpCgpzZXFJcGxvdChsb2dfc3RzLnNlcVsxOjEwMDAsMToyMDBdLCBncm91cCA9IGNvdmFyJGNvbmRpdGlvbiwgc29ydHYgPSBkaXN0LnJlZnNlcSwgd2l0aC5sZWdlbmQgPSAicmlnaHQiKQpgYGAKCgoK